{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 112,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.metrics import log_loss  \n",
    "from matplotlib import pyplot\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "读取文件"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 113,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "      <th>Outcome</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  Outcome  \n",
       "0                     0.627   50        1  \n",
       "1                     0.351   31        0  \n",
       "2                     0.672   32        1  \n",
       "3                     0.167   21        0  \n",
       "4                     2.288   33        1  "
      ]
     },
     "execution_count": 113,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train=pd.read_csv(\"diabetes.csv\")\n",
    "train.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 768 entries, 0 to 767\n",
      "Data columns (total 9 columns):\n",
      "Pregnancies                 768 non-null int64\n",
      "Glucose                     768 non-null int64\n",
      "BloodPressure               768 non-null int64\n",
      "SkinThickness               768 non-null int64\n",
      "Insulin                     768 non-null int64\n",
      "BMI                         768 non-null float64\n",
      "DiabetesPedigreeFunction    768 non-null float64\n",
      "Age                         768 non-null int64\n",
      "Outcome                     768 non-null int64\n",
      "dtypes: float64(2), int64(7)\n",
      "memory usage: 54.1 KB\n"
     ]
    }
   ],
   "source": [
    "train.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 115,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>6</td>\n",
       "      <td>148</td>\n",
       "      <td>72</td>\n",
       "      <td>35</td>\n",
       "      <td>0</td>\n",
       "      <td>33.6</td>\n",
       "      <td>0.627</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>85</td>\n",
       "      <td>66</td>\n",
       "      <td>29</td>\n",
       "      <td>0</td>\n",
       "      <td>26.6</td>\n",
       "      <td>0.351</td>\n",
       "      <td>31</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>8</td>\n",
       "      <td>183</td>\n",
       "      <td>64</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>23.3</td>\n",
       "      <td>0.672</td>\n",
       "      <td>32</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>89</td>\n",
       "      <td>66</td>\n",
       "      <td>23</td>\n",
       "      <td>94</td>\n",
       "      <td>28.1</td>\n",
       "      <td>0.167</td>\n",
       "      <td>21</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>137</td>\n",
       "      <td>40</td>\n",
       "      <td>35</td>\n",
       "      <td>168</td>\n",
       "      <td>43.1</td>\n",
       "      <td>2.288</td>\n",
       "      <td>33</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pregnancies  Glucose  BloodPressure  SkinThickness  Insulin   BMI  \\\n",
       "0            6      148             72             35        0  33.6   \n",
       "1            1       85             66             29        0  26.6   \n",
       "2            8      183             64              0        0  23.3   \n",
       "3            1       89             66             23       94  28.1   \n",
       "4            0      137             40             35      168  43.1   \n",
       "\n",
       "   DiabetesPedigreeFunction  Age  \n",
       "0                     0.627   50  \n",
       "1                     0.351   31  \n",
       "2                     0.672   32  \n",
       "3                     0.167   21  \n",
       "4                     2.288   33  "
      ]
     },
     "execution_count": 115,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "train_y=train['Outcome']\n",
    "train_x=train.drop([\"Outcome\"], axis=1)\n",
    "train_y.head()\n",
    "train_x.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 116,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pregnancies</th>\n",
       "      <th>Glucose</th>\n",
       "      <th>BloodPressure</th>\n",
       "      <th>SkinThickness</th>\n",
       "      <th>Insulin</th>\n",
       "      <th>BMI</th>\n",
       "      <th>DiabetesPedigreeFunction</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "      <td>768.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>3.845052</td>\n",
       "      <td>120.894531</td>\n",
       "      <td>69.105469</td>\n",
       "      <td>20.536458</td>\n",
       "      <td>79.799479</td>\n",
       "      <td>31.992578</td>\n",
       "      <td>0.471876</td>\n",
       "      <td>33.240885</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>3.369578</td>\n",
       "      <td>31.972618</td>\n",
       "      <td>19.355807</td>\n",
       "      <td>15.952218</td>\n",
       "      <td>115.244002</td>\n",
       "      <td>7.884160</td>\n",
       "      <td>0.331329</td>\n",
       "      <td>11.760232</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078000</td>\n",
       "      <td>21.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>62.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>27.300000</td>\n",
       "      <td>0.243750</td>\n",
       "      <td>24.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>3.000000</td>\n",
       "      <td>117.000000</td>\n",
       "      <td>72.000000</td>\n",
       "      <td>23.000000</td>\n",
       "      <td>30.500000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>0.372500</td>\n",
       "      <td>29.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>6.000000</td>\n",
       "      <td>140.250000</td>\n",
       "      <td>80.000000</td>\n",
       "      <td>32.000000</td>\n",
       "      <td>127.250000</td>\n",
       "      <td>36.600000</td>\n",
       "      <td>0.626250</td>\n",
       "      <td>41.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>17.000000</td>\n",
       "      <td>199.000000</td>\n",
       "      <td>122.000000</td>\n",
       "      <td>99.000000</td>\n",
       "      <td>846.000000</td>\n",
       "      <td>67.100000</td>\n",
       "      <td>2.420000</td>\n",
       "      <td>81.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "       Pregnancies     Glucose  BloodPressure  SkinThickness     Insulin  \\\n",
       "count   768.000000  768.000000     768.000000     768.000000  768.000000   \n",
       "mean      3.845052  120.894531      69.105469      20.536458   79.799479   \n",
       "std       3.369578   31.972618      19.355807      15.952218  115.244002   \n",
       "min       0.000000    0.000000       0.000000       0.000000    0.000000   \n",
       "25%       1.000000   99.000000      62.000000       0.000000    0.000000   \n",
       "50%       3.000000  117.000000      72.000000      23.000000   30.500000   \n",
       "75%       6.000000  140.250000      80.000000      32.000000  127.250000   \n",
       "max      17.000000  199.000000     122.000000      99.000000  846.000000   \n",
       "\n",
       "              BMI  DiabetesPedigreeFunction         Age  \n",
       "count  768.000000                768.000000  768.000000  \n",
       "mean    31.992578                  0.471876   33.240885  \n",
       "std      7.884160                  0.331329   11.760232  \n",
       "min      0.000000                  0.078000   21.000000  \n",
       "25%     27.300000                  0.243750   24.000000  \n",
       "50%     32.000000                  0.372500   29.000000  \n",
       "75%     36.600000                  0.626250   41.000000  \n",
       "max     67.100000                  2.420000   81.000000  "
      ]
     },
     "execution_count": 116,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_x.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 117,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# 初始化特征的标准化器\n",
    "ss_X = StandardScaler()\n",
    "\n",
    "# 分别对训练和测试数据的特征进行标准化处理\n",
    "train_x = ss_X.fit_transform(train_x)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import train_test_split\n",
    "train_x_part, test_x, train_y_part, test_y = train_test_split(train_x, train_y,test_size=0.2,random_state = 0)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.linear_model import LogisticRegression\n",
    "def fit_grid_point_LR(penalty, C, train_x, train_y, test_x, test_y):\n",
    "    \n",
    "    # 在训练集训练LR\n",
    "    LR = LogisticRegression(penalty=penalty, C=C)\n",
    "    LR.fit(train_x, train_y)\n",
    "    \n",
    "    # 在训练集和校验集上测试\n",
    "    y_train_pred = LR.predict_proba(train_x)\n",
    "    y_test_pred = LR.predict_proba(test_x)\n",
    "    logloss_test = log_loss(test_y,y_test_pred)\n",
    "    logloss_train = log_loss(train_y, y_train_pred)\n",
    "    \n",
    "    print(\"logloss on test: %f and on train: %f with C = %f and penalty = %s\"%(logloss_test, logloss_train, C, penalty) )\n",
    "    return logloss_test"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "logloss on test: 0.693147 and on train: 0.693147 with C = 0.001000 and penalty = l1\n",
      "logloss on test: 0.630865 and on train: 0.638772 with C = 0.001000 and penalty = l2\n",
      "logloss on test: 0.645693 and on train: 0.648942 with C = 0.010000 and penalty = l1\n",
      "logloss on test: 0.499843 and on train: 0.528853 with C = 0.010000 and penalty = l2\n",
      "logloss on test: 0.444972 and on train: 0.492139 with C = 0.100000 and penalty = l1\n",
      "logloss on test: 0.437743 and on train: 0.485622 with C = 0.100000 and penalty = l2\n",
      "logloss on test: 0.429155 and on train: 0.483175 with C = 1.000000 and penalty = l1\n",
      "logloss on test: 0.429425 and on train: 0.483079 with C = 1.000000 and penalty = l2\n",
      "logloss on test: 0.428596 and on train: 0.483041 with C = 10.000000 and penalty = l1\n",
      "logloss on test: 0.428633 and on train: 0.483040 with C = 10.000000 and penalty = l2\n",
      "logloss on test: 0.428558 and on train: 0.483040 with C = 100.000000 and penalty = l1\n",
      "logloss on test: 0.428555 and on train: 0.483040 with C = 100.000000 and penalty = l2\n",
      "logloss on test: 0.428545 and on train: 0.483040 with C = 1000.000000 and penalty = l1\n",
      "logloss on test: 0.428547 and on train: 0.483040 with C = 1000.000000 and penalty = l2\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "\n",
    "logloss_s = []\n",
    "for i, OneC in enumerate(Cs):\n",
    "    for j, onePenalty in enumerate(penaltys):\n",
    "        tmp = fit_grid_point_LR(onePenalty, OneC, train_x_part, train_y_part, test_x, test_y)\n",
    "        logloss_s.append(tmp)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYsAAAEKCAYAAADjDHn2AAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzt3Xl8VPW9//HXZ7IQIIEQCBAIEFBAtgAaQFCoYhWEVG21VWpvtbW13l6X2l6rvbXXe7W2dvm12lvbq1Vr67VStVoVUNxXXAgqIIuKyBJlCfsaIMnn98dMMIRAJslMzszk/Xw85pGZM+fM+RyWeed8z/d8v+buiIiIHE0o6AJERCTxKSxERKRRCgsREWmUwkJERBqlsBARkUYpLEREpFEKCxERaZTCQkREGqWwEBGRRqUHXUCsdOvWzYuKioIuQ0QkqSxYsGCTu+c3tl5cw8LMpgK3AWnAXe5+S733fwucGnnZAeju7rmR9y4Cro+891N3/8vR9lVUVERZWVksyxcRSXlmtjqa9eIWFmaWBtwOnA6UA/PN7HF3X1q7jrtfXWf9K4DRked5wA1ACeDAgsi2W+NVr4iIHFk8r1mMBVa4+0p33w/MBM4+yvozgAciz6cAz7j7lkhAPANMjWOtIiJyFPEMi97A2jqvyyPLDmNm/YD+wPNN3VZEROIvntcsrIFlRxoP/QLgYXevbsq2ZnYpcClA3759m1OjiKSIAwcOUF5eTmVlZdClJKSsrCwKCwvJyMho1vbxDItyoE+d14XAp0dY9wLg3+pte0q9bV+sv5G73wncCVBSUqKJOUTasPLycnJycigqKsKsod832y53Z/PmzZSXl9O/f/9mfUY8m6HmAwPNrL+ZZRIOhMfrr2Rmg4EuwOt1Fs8FzjCzLmbWBTgjskxEpEGVlZV07dpVQdEAM6Nr164tOuuK25mFu1eZ2eWEv+TTgHvcfYmZ3QiUuXttcMwAZnqdKfvcfYuZ3UQ4cABudPct8apVRFKDguLIWvpnE9f7LNx9DjCn3rL/rPf6v46w7T3APXErLqKmxrnlqeV8dWxfirp1jPfuRESSUpsf7mPV5t3MfGsN0373Cg+8tQbNSS4iLTVu3DhGjRpF3759yc/PZ9SoUYwaNYpVq1Y16XMeeeQRli9f3uB7VVVV5ObmAlBTU8OUKVPIzc3lnHPOaWn5DUqZ4T6aa0B+Nk99bxL//tBCfvTIYp5btpFbzh1Bt+x2QZcmIknqzTffBODee++lrKyM3//+9836nEceeYRQKMRxxx131PXMjB/+8Ifs3LmTe++9t1n7akybP7MA6JXbnv+7ZBzXTx/Cyx9WMPXWl3l26YagyxKRFPTkk08yfvx4jj/+eM4//3x2794NwDXXXMPQoUMpLi7m2muv5ZVXXmHOnDlcffXVjZ6VmBmnnXYa2dnZcau7zZ9Z1AqFjG9NHMDEgflcNfMdvvXXMmaM7cv104fQsZ3+mESSyX8/sYSln+6I6WcO7dWJG74wrEWfsXHjRm655Raee+45OnTowM0338xtt93GJZdcwpw5c1iyZAlmxrZt28jNzWXatGmcd955cWtaagqdWdQzuGcOj11+Et/53ABmzl/D9N+9wjtrNCSViLTcvHnzWLp0KRMmTGDUqFHcf//9rFq1iry8PEKhEN/+9rd59NFH6dgx8Trb6FfmBrRLT+NHZw7h1MHd+cGDCznvf1/n8lOP5fLJx5KRpnwVSXQtPQOIF3dn6tSp3HfffYe9V1ZWxjPPPMPMmTP54x//yNNPP33I+6tWrTp4hnH55Zdz8cUXt0bJByksjuLEAV158nsT+a/HlnDbcx/y4gcV/PYrIxmQH792QRFJXRMmTOCqq65i5cqVDBgwgN27d/Ppp5/Ss2dPKisrKS0tZdy4cQwdOhSAnJwcdu7cCYSnYXj33XcPflZVVVWr1q5fkxvRKSuD35w/itu/ejyrNu1m+u9e5f/eWK0utiLSZD169ODuu+/m/PPPZ+TIkUyYMIEPPviA7du3M336dEaOHMnkyZP5zW9+A8CMGTP42c9+FlW32/HjxzNjxgzmzp1LYWEhzz33XExrt1T50ispKfF4T360fnsl1zy8kFc+3MTk47rzi3OLyc9RF1uRRLBs2TKGDBkSdBkJraE/IzNb4O4ljW2rM4sm6Nk5i798Yyw3fGEor63YxJRbX+bpJeuDLktEJO4UFk0UChnfOKk/s644mYLOWVx63wKufXgRu/a1bvuhiEhrUlg008AeOTz63ZP47inH8OCCtUy77RUWrFYXWxFJTQqLFshMD/HDqcfx4HfGU+POl/93Hv/v6fc5UF0TdGkiIjGlsIiBMUV5PHnVRL50fCH/8/wKzv3jPD6q2BV0WSIiMaOwiJGcrAx+/eWR/O/Xjmftlj1M/90r/PX1VepiKyIpQWERY1OHFzD3e5MY178r//nYEi7+83w27tCcwCJtSWsPUb5gwQJOPPFEhg8fTnFxMQ8//HBLD+EwuoM7Drp3yuLeb4zhvjdW87M5y5hy68v8/EvFTB3eM+jSRKQVtPYQ5dnZ2dx///0cc8wxlJeXU1JSwpQpU8jJyWnWfhuiM4s4MTO+Pr6IWVdMpLBLBy77vwVc89BCdlYeCLo0EQlQPIYoHzx4MMcccwwAhYWFdO3alU2bNsW0bp1ZxNmx3bN55LsT+N1zH3L7Cyt4feVmfnv+KMYU5QVdmkjqevI6WL84tp/ZcwSceUuLPqI1hiifN28eEB5LKpZ0ZtEKMtJC/OCMwTx02XhCZpx/x+v88qnl7K9SF1uRtiTeQ5R/8sknXHzxxdx7772YWUxr15lFKzqhXx5zrprITU8s5Q8vfsTLH1Zw6/mjOLZ77NoVRYQWnwHESzyHKK8djPAXv/gFY8aMiXntCotWlt0unV+cV8zkId350SOLmf67V/nRmcfx9fFFhEKx/U1ARBJLvIYo37dvH2effTaXXHIJX/ziF+NSu5qhAjJlWE+e+t5ETjq2G//1xFIu+vNbbFAXW5GUFq8hyh944AHmzZvH3XfffbCb7uLFsb1moyHKA+bu/O2tNfx01jLaZYT42RdHMG1EQdBliSQdDVHeOA1RnsTMjAvH9WP2lSfTL68D373/bb7/4LvsUBdbEUkgCosEMSA/m4f/dQJXnTaQx979lDNvfYU3V24OuiwREUBhkVAy0kJcffogHr5sPBlpxgV/eoOfP7mMfVXVQZcmkhRSpVk9Hlr6Z6OwSECj+3Zh9pUTuWBMX+54aSXn3D6PDzbsDLoskYSWlZXF5s2bFRgNcHc2b95MVlZWsz9DF7gT3LNLN3DdI4vYUVnFtVOP4xsT1MVWpCEHDhygvLycykr1KmxIVlYWhYWFZGRkHLI82gvccQ0LM5sK3AakAXe5+2F3ypjZV4D/AhxY6O5fjSyvBmr7fq1x97OOtq9UDQuATbv2cd0/FvHsso2cdGxXfv3lkRR0bh90WSKSAgIPCzNLAz4ATgfKgfnADHdfWmedgcCDwGR332pm3d19Y+S9Xe6eHe3+UjksIHwaOXP+Wm6atZT0kHHzF0fwhZG9gi5LRJJcInSdHQuscPeV7r4fmAmcXW+dbwO3u/tWgNqgkMOZGTPG9mXOlRM5pns2VzzwDt+b+Q7b96qLrYjEXzzDojewts7r8siyugYBg8zsNTN7I9JsVSvLzMoiyxscctHMLo2sU1ZRURHb6hNUUbeOPPSd8Xz/9EE8sWgdZ976MvM+iu1QxCIi9cUzLBq6Clu/zSsdGAicAswA7jKz3Mh7fSOnRl8FbjWzYw77MPc73b3E3Uvy8/NjV3mCS08LceVpA3nkXyeQlZHGhXe9yc/mqIutiMRPPMOiHOhT53Uh8GkD6zzm7gfc/WPgfcLhgbt/Gvm5EngRGB3HWpPSyD65zLryZC4c15c7X17J2b9/jeXrdwRdloikoHiGxXxgoJn1N7NM4ALg8Xrr/BM4FcDMuhFullppZl3MrF2d5ScBS5HDdMhM56fnjODPF49h0679nHP7a3yybW/QZYlIiolbWLh7FXA5MBdYBjzo7kvM7EYzq+0GOxfYbGZLgReAa9x9MzAEKDOzhZHlt9TtRSWHO/W47jx02XgqD9Qwa2H9EzgRkZbRTXkp5uzbX6OmxnniipODLkVEkkAidJ2VAJSOKGDxJ9tZtWl30KWISApRWKSYacXhuTBmL14XcCUikkoUFimmd257ju+by6xFCgsRiR2FRQoqLe7FsnU7+KhiV9CliEiKUFikoGkjCjCD2Tq7EJEYUVikoJ6dsxjTL49Zi9SFVkRiQ2GRokpHFvDBhl2aNElEYkJhAVC5Haqrgq4ipqYO70nI0IVuEYkJhcXmj+DWYlj8YNCVxFT3nCzG9e/KrEWfappJEWkxhUXeAOjSD168Bar2B11NTJWOLGBlxW6Wr1dTlIi0jMLCDCb/BLathnfuC7qamJo6rCdpIdOFbhFpMYUFwLGfhz7j4OVfwYHUGbG1a3Y7JhzTldmL1qkpSkRaRGEBn51d7FwHZfcEXU1MTR9RwKrNe1jyqea5EJHmU1jU6j8RBpwCr/wG9qXOnc9Th/ckPWQ8oaYoEWkBhUVdk38CezbBm38MupKYye2QyckDu6kpSkRaRGFRV2EJDDoTXvsf2Ls16GpiZvqIAsq37mVh+fagSxGRJKWwqG/yj2Hfdpj3+6AriZkzhvUkI82YraYoEWkmhUV9PUfAsC/CG3+EXRVBVxMTndtnMGlgPrMXraOmRk1RItJ0CouGnPIfULUXXv1t0JXETOnIAj7dXsk7a1OneU1EWo/CoiH5g2DkDJh/F+xIjaabzw/pQWZ6SGNFiUizKCyO5HPXgteEb9RLATlZGZwyKJ85i9UUJSJNp7A4ki794Pivw9t/hS0fB11NTJSO7MWGHfsoW62mKBFpGoXF0Uy6BkLp8NIvg64kJk47rjtZGSGNFSUiTaawOJpOBTDmW7BoJlS8H3Q1LdaxXTqTj+vOnMXrqVZTlIg0gcKiMSdfDRkd4IWfBV1JTJQW92LTrn28+fHmoEsRkSSisGhMx25w4r/C0n/CukVBV9Nipw7uTofMNPWKEpEmUVhEY/zlkNUZXrg56EparH1mGqcN6cFT762nqrom6HJEJEkoLKLRPhcmXAkfPAVr5wddTYtNH1HAlt37eX2lmqJEJDpxDQszm2pm75vZCjO77gjrfMXMlprZEjP7W53lF5nZh5HHRfGsMyrjLoOO+fD8TUFX0mKnDM4nu106sxaqKUpEohO3sDCzNOB24ExgKDDDzIbWW2cg8CPgJHcfBnwvsjwPuAEYB4wFbjCzLvGqNSrtsuHk78PHL8HHLwdaSktlZaRx+tAePLVkPQfUFCUiUYjnmcVYYIW7r3T3/cBM4Ox663wbuN3dtwK4+8bI8inAM+6+JfLeM8DUONYanZJvQk4veO4mSPK5IaaPKGD73gO8umJT0KWISBKIZ1j0BtbWeV0eWVbXIGCQmb1mZm+Y2dQmbNv6MrLgc9dA+Vvw4TNBV9MiEwd1IycrndnqFSUiUYhnWFgDy+r/Op4ODAROAWYAd5lZbpTbYmaXmlmZmZVVVLTScOKj/wW6FIWvXdQkbxNOu/Q0pgzrydwl69lXVR10OSKS4OIZFuVAnzqvC4H640yUA4+5+wF3/xh4n3B4RLMt7n6nu5e4e0l+fn5Miz+itAz43HWwfhEse7x19hkn04sL2FlZxSsfqClKRI4unmExHxhoZv3NLBO4AKj/7fpP4FQAM+tGuFlqJTAXOMPMukQubJ8RWZYYir8C3QaH7+quSd7fyk8+thud22cwe7GaokTk6OIWFu5eBVxO+Et+GfCguy8xsxvN7KzIanOBzWa2FHgBuMbdN7v7FuAmwoEzH7gxsiwxhNLg1P+ATe/D4oeCrqbZMtJCTB3Wk2eWbqDyQPKGnojEn3mS9+qpVVJS4mVlZa23w5oauHMS7NsJl5eFm6eS0CsfVvAvd7/FHf9yAlOG9Qy6HBFpZWa2wN1LGltPd3A3VygEk38CW1fBO/8XdDXNNn5AV/I6ZmqsKBE5KoVFSww8AwrHhmfTO1AZdDXNkp4WYurwnjy3bAN796spSkQaprBoCTOYfD3s+ATK7gm6mmYrLS5gz/5qXnh/Y+Mri0ibpLBoqQGfg/6T4NXfwL5dQVfTLOP6d6VbdjvNoCciR6SwiIXJP4HdFfDWHUFX0ixpIWPaiJ48v3wju/dVBV2OiCQghUUs9BkLA6fAa7fB3m1BV9MspcW9qDxQw3PL1RQlIodTWMTK5Ouhcju8fnvQlTRLSb8u9OjUjlkL1RQlIodTWMRKQTEMPQfe+APsTr7hM0IhY9qIAl78oIKdlQeCLkdEEozCIpZO/Q84sAde/W3QlTRLaXEB+6tqeHbZhqBLEZEE0+SwMLOQmXWKRzFJL38wFJ8P8++CHcl3k9voPl3o1TlLw5aLyGGiCgsz+5uZdTKzjsBS4H0zuya+pSWpz10LNVXwyq+DrqTJQiFjenEBL31Qwfa9aooSkc9Ee2Yx1N13AOcAc4C+wL/Erapkltc/POfFgr/A1tVBV9Nk04t7caDaeXrJ+qBLEZEEEm1YZJhZBuGweMzdD9DAZEQSMekasBC89MugK2mykYWdKezSXsOWi8ghog2LO4BVQEfgZTPrB+yIV1FJr3NvGPMtWPg32PRh0NU0iVm4KerVDzexdff+oMsRkQQRVVi4++/cvbe7T/Ow1UQmLZIjOPlqSG8fniApyXyhuBdVNc7TS9UUJSJh0V7gvipygdvM7G4zexuYHOfaklt2Ppx4GSx5BNa/F3Q1TTKsVyeKunbQsOUiclC0zVDfjFzgPgPIB74B3BK3qlLFhCugXWd44eagK2mS2qaoeR9tZvOufUGXIyIJINqwsMjPacCf3X1hnWVyJO27wElXwPtzoLwVZ/GLgdLiXlTXOE+pV5SIEH1YLDCzpwmHxVwzywFq4ldWChl3GXToCs//NOhKmuS4njkMyO/IrIVqihKR6MPiEuA6YIy77wEyCTdFSWPa5cDJ34eVL8CqV4OuJmpmRmlxL978eDMbdybnLIAiEjvR9oaqAQqB683s18AEd18U18pSyZhLIKcAnrsJPHluTyktLqDG4an31BQl0tZF2xvqFuAqwkN9LAWuNLOfx7OwlJLRHib9O6x9A1Y8F3Q1URvUI4dBPbLVFCUiUTdDTQNOd/d73P0eYCowPX5lpaDRX4fcvvB8sp1d9GL+6i2s366mKJG2rCmjzubWed451oWkvPRMOOVHsO5dWPZE0NVEbXpxAe4wR8N/iLRp0YbFz4F3zOxeM/sLsABIvluTgzbiK9B1YPiu7prqoKuJyjH52Qwp6MSsRZpBT6Qti/YC9wPAicAjkcd4d58Zz8JSUlp6eIKkimXw3j+CriZqpcUFvL1mG59s2xt0KSISkKOGhZkdX/sACoByYC3QK7JMmmroOdBjRPjsojo55owoLS4AYI6G/xBps9Ibef//HeU9R+NDNV0oBJN/DA9cAO/+DU64KOiKGtWva0dG9O7MrMXr+PakAUGXIyIBOGpYuLtGlo2HQVOhd0l4vouRF0B6u6AratT04gJueXI5a7fsoU9eh6DLEZFWFu19Fl9q4HGamXVvZLupZva+ma0ws+saeP9iM6sws3cjj2/Vea+6zvLHm35oCcwMTvsJ7CiHsj8HXU1Upo8IN0VpJFqRtqkpw33cBVwYefwJ+D7wmpk1OL2qmaUBtwNnAkOBGWY2tIFV/+7uoyKPu+os31tn+VlR1pk8+n8OiiaG5+revzvoahrVJ68Do/rkMnuxekWJtEXRhkUNMMTdz3X3cwl/+e8DxgHXHmGbscAKd1/p7vuBmcDZLS04ZZjB5J/A7gp4686gq4lKaXEB732yg1WbEj/cRCS2og2LInffUOf1RmCQu28BjtSlpzfhnlO1yiPL6jvXzBaZ2cNm1qfO8iwzKzOzN8zsnIZ2YGaXRtYpq6ioiPJQEkjfcTDwDHj1VqjcHnQ1jZoWaYrS/NwibU+0YfGKmc0ys4vM7CLgccJzcXcEth1hm4bmu6g/zsUThIOoGHgW+Eud9/q6ewnwVeBWMzvmsA9zv9PdS9y9JD8/P8pDSTCn/hgqt8HrtwddSaN65bbnhH5deGKhmqJE2ppow+LfgD8Do4DRhL/U/83ddx+lx1Q5UPdMoRA45FvG3Te7e+1UbH8CTqjz3qeRnyuBFyP7TT29RsGQs+D1P8DuzUFX06jS4gKWr9/Jio27gi5FRFpRtHdwO/Aq8DzhM4CXI8uOZj4w0Mz6m1kmcAHhM5KDzKygzsuzgGWR5V3MrF3keTfgJMKj3aamU38M+3fBa7cGXUmjpo0owAxmq1eUSJsSbdfZrwBvAecBXwHeNLPzjraNu1cBlwNzCYfAg+6+xMxuNLPa3k1XmtkSM1sIXAlcHFk+BCiLLH8BuMXdUzcsuh8HxV+Bt/4EOxN77ogenbIYU5SnsaJE2hhr/AQBIl/ap7v7xsjrfOBZdx8Z5/qiVlJS4mVlyTXP9SG2rITfj4GSb8K0XwVdzVHd9/oqfvLYEp6+ehKDeuQEXY6ItICZLYhcHz6qaK9ZhGqDImJzE7aVaOQNgNFfC9+kt21N0NUc1dThBYQMZulCt0ibEe0X/lNmNjdyx/XFwGxgTvzKaqMmXRO+/+KlXwRdyVHl57TjxAFdmbVoHdGcmYpI8ov2Avc1wJ1AMTASuNPdj3QznjRX50IouQTefQA2rQi6mqMqLe7Fyk27WbZuZ9CliEgriLopyd3/4e7fd/er3f3ReBbVpk38fnhgwRcTe4rzqcN7khYyXegWaSMam89ip5ntaOCx08x2tFaRbUp2dxh3WXhypA1Lgq7miPI6ZjLhmK7MXqymKJG24Khh4e457t6pgUeOu3dqrSLbnAlXQLuc8ARJCewLxb1YvXkP732i3xtEUp16NCWiDnnhwFg+Cz5ZEHQ1R3TGsB6kqylKpE1QWCSqE/8V2ufB8z8NupIjyu2QycSB3dQrSqQNUFgkqnY5cPLV8NHzsOq1oKs5ounFvfhk217eXXuk8SRFJBUoLBLZ2G9Dds/w2UWC/uZ+xrAeZKaFNFaUSIpTWCSyjPYw6d9hzTz46Lmgq2lQp6wMJg3KZ/biddTUJGagiUjLKSwS3fEXQee+CX12UVpcwLrtlby9ZmvQpYhInCgsEl16JpxyLXz6DiyfHXQ1Dfr80B5kpoeYpaYokZSlsEgGxRdA12PhhZuhpjroag6T3S6dUwfnM2fxOqrVFCWSkhQWySAtHU75EWxcCksSc6SV0uJebNy5j7JVW4IuRUTiQGGRLIZ9CXoMD9/VXV0VdDWHOW1Id7Iy1BQlkqoUFskiFApPv7rlI1j4t6CrOUyHzHROO64HT763jqrqmqDLEZEYU1gkk8FnQu8T4KVfQtW+oKs5TGlxAZt27eetj9UUJZJqFBbJxAwmXw/b18KCvwRdzWFOGdydDplpPKGmKJGUo7BINgNOhX4nwyu/hv17gq7mEO0z0/j8kB48paYokZSjsEg2tWcXuzbA/D8FXc1hSosL2LrnAPM+2hx0KSISQwqLZNRvPBz7eXj1t1CZWHNJTBqUT067dA1bLpJiFBbJavL1sHcrvPGHoCs5RFZGGqcP7cHcJRvYX6WmKJFUobBIVr1Gw3Gl8PrtsCexeh9NLy5g+94DvLZiU9CliEiMKCyS2eTrYd9OeO22oCs5xMSB+XTKStcNeiIpRGGRzLoPgRFfhjfvgJ0bgq7moMz0EFOG9eTppevZV5V4Y1mJSNMpLJLdKddB9X549TdBV3KI6cUF7Kys4uUP1BQlkgoUFsmu6zEw+kIouwe2rQ26moNOOrYbuR0ymK1eUSIpIa5hYWZTzex9M1thZtc18P7FZlZhZu9GHt+q895FZvZh5HFRPOtMepN+GP758i+DraOOjLQQU4f15JmlG6g8oKYokWQXt7AwszTgduBMYCgww8yGNrDq3919VORxV2TbPOAGYBwwFrjBzLrEq9akl9sHTvgGvHM/bP4o6GoOKi3uxe791bz4fkXQpYhIC8XzzGIssMLdV7r7fmAmcHaU204BnnH3Le6+FXgGmBqnOlPDxB9AWia8eEvQlRx04oA8unbM1A16IikgnmHRG6jbiF4eWVbfuWa2yMweNrM+TdxWauX0gHHfgcUPwcK/B10NAOlpIaYO78lzyzayZ3/izcEhItGLZ1hYA8vqz7n5BFDk7sXAs0DtUKrRbIuZXWpmZWZWVlGhpg4mXQP9J8Kjl8LriXFnd2lxL/YeqOaF5fr7EUlm8QyLcqBPndeFwCHtEe6+2d1rJ2b4E3BCtNtGtr/T3UvcvSQ/Pz9mhSetdtnw1YdgyFkw90fw7H+DBzsn9tj+eeTntFNTlEiSi2dYzAcGmll/M8sELgAer7uCmRXUeXkWsCzyfC5whpl1iVzYPiOyTBqTkQVfvjd8wfvV38DjVwQ6DWtayJg2vCfPL9/I7n1qihJJVnELC3evAi4n/CW/DHjQ3ZeY2Y1mdlZktSvNbImZLQSuBC6ObLsFuIlw4MwHbowsk2iE0qD0t+Eute/cBw9dBAcqAyundGQv9lXV8OyyxLnLXESaxjzgZopYKSkp8bKysqDLSDxv3gFP/jA8YdKMv0FW51YvoabGmXDL84wo7Myfvl7S6vsXkSMzswXu3uh/TN3BnerGfQfOvRvWvgH3ToddG1u9hFDImDaigJfer2Bn5YFW37+ItJzCoi0YcR589e/hG/buPgO2fNzqJUwvLmB/dQ3PLFVTlEgyUli0Fcd+Hi56Aiq3wT1TYP3iVt398X1z6Z3bXsOWiyQphUVbUlgC35wLoXT48zRY9Vqr7drMmF5cwCsfVrB9j5qiRJKNwqKtyR8MlzwNOT3hvi/C8tmttuvpIwo4UO3MXbq+1fYpIrGhsGiLOheGzzB6Doe/fw3evq9Vdltc2Jm+eR2YraYokaSjsGirOuTB1x+HAafA45fDq7fG/W7v2qao11ZsYuvu/XHdl4jElsKiLWuXDTP+DsPPhWdvgKevh5qauO5y+ogCqmqcp5aoKUokmSgs2rr0TPjSXTD2Unh3irA3AAAO8ElEQVT99/DYd6E6fhegh/XqRP9uHdUUJZJkFBYCoRCc+Us49XpY+ADMvBD274nLrsyM6SMKmPfRJjbt2tf4BiKSEBQWEmYGn7smPKbUh0/DfefA3q1x2VXpyAJqHJ56T01RIslCYSGHKvlmeNTaT98J34uxI/ZDiw/ukcOx3bM1bLlIElFYyOGGnQMXPgzb1sDdU2DTiph+fG1T1Jsfb2HjjuBGwxWR6CkspGEDPgcXz4IDe+CeM8JnGjFUWlyAOzyppiiRpKCwkCPrNTp8t3dmR7i3FFa+FLOPHtgjh8E9ctQUJZIkFBZydF2PgW8+Dbl94f7zYMk/Y/bRpcUFzF+1lfXb1RQlkugUFtK4TgXwjTnQ63h46GKYf3dMPnZ6cXhW3dmLdc+FSKJTWEh02neBf3kUBp4Bs78PL/2yxcODDMjPZmhBJzVFiSQBhYVEL7MDXHA/jJwBL9wcnq61hcODlI4s4J012yjfGp+bAEUkNhQW0jRpGXD2H2D85fDWnfDIt6Cq+YMClo7oBcAcNUWJJDSFhTRdKARTbobTb4T3/gEPnA/7djXro/p27UBxYWeNFSWS4BQW0nwnXQVn3w4rX4S/ngW7NzfrY0qLC1hYvp01m9UUJZKoFBbSMqO/BuffDxuWhOf23ra2yR8xbUS4V9SsxbrQLZKoFBbScsdNg689Ars2hgNj4/ImbV7YpQOj++aqKUokgSksJDaKToJvzIaaKvjzVFg7v0mbTx9RwJJPd/Dxpt1xKlBEWkJhIbHTc0R4bu+s3PA1jA+fjXrTgzfo6Z4LkYSksJDYyusfHk+q6zHhXlKLHopqs4LO7RlT1IVZaooSSUgKC4m97O5w8Wzoc2L4Pow3/jeqzaaPKGD5+p2s2LgzzgWKSFPFNSzMbKqZvW9mK8zsuqOsd56ZuZmVRF4XmdleM3s38oju20YSR1Zn+No/4LhSeOpaeP6njQ4PMm1EAWbo7EIkAcUtLMwsDbgdOBMYCswws6ENrJcDXAm8We+tj9x9VORxWbzqlDjKyIIv/wWO/zq8/CuY9T2oqT7i6t07ZTG2KI9Zi9bhLRx3SkRiK55nFmOBFe6+0t33AzOBsxtY7ybgl4DGqU5Faenwhd/BxB/AgnvhoYvgwJH/qktH9mLFxl18sKF5d4SLSHzEMyx6A3Xv0CqPLDvIzEYDfdx9VgPb9zezd8zsJTObGMc6Jd7M4LT/hCk/h2VPhOfFqNzR4KpnDu9JyNBItCIJJp5hYQ0sO9i2YGYh4LfADxpYbx3Q191HA98H/mZmnQ7bgdmlZlZmZmUVFRUxKlviZvx34Ut/gjWvw73Twzfx1dMtux3jj+mqpiiRBBPPsCgH+tR5XQjU/XUxBxgOvGhmq4ATgcfNrMTd97n7ZgB3XwB8BAyqvwN3v9PdS9y9JD8/P06HITFV/BWYMRM2fRi+23vrqsNWKS3uxcebdrN0XcNnHyLS+uIZFvOBgWbW38wygQuAx2vfdPft7t7N3YvcvQh4AzjL3cvMLD9ygRwzGwAMBFbGsVZpTQNPh4sehz1b4O4zYP17h7w9ZVhP0kKmXlEiCSRuYeHuVcDlwFxgGfCguy8xsxvN7KxGNp8ELDKzhcDDwGXuviVetUoA+oyFbz4FlgZ/ngar5x18K69jJicf2407XvqIabe9wg2PvccTCz9l3fa9ARYs0rZZqrQLl5SUeFlZWdBlSFNtWwv3fRG2r4Uv3wuDzwRg445K7n9zDWWrt/DOmm3s2R/ucts7N3yn9wlFeYwp6sKg7jmEQg1dHhORaJjZAncvaXQ9hYUEbvemcA+pdYvgrP+B0Rce8nZVdQ3L1u1k/qotlK3ewvxVW6nYuQ+AnKx0TujXhTFFeZzQrwuj+uSSlZEWxFGIJCWFhSSXfTvh718LT6R0+o3hiZWOwN1Zu2VvJDy2UrZqCx9uDN+XkZFmDO/dmZJ+XSgpyqOkXxe6ZrdrpYMQST4KC0k+Vfvg0e/AkkdhwhVw+k3hezSisHX3ft5es5X5q8Lhsah8O/urawAY0K0jJUWfhUf/bh2xKD9XJNUpLCQ51VTDkz+E+XdB0UToNQo694XcvpDbBzr3gazDbrk5TOWBat77ZPvBM4+y1VvZtucAAF07ZobDo18eJUVdGNarM5npGlNT2iaFhSQvd3jtNnjnPtheDlX1hgfJyg0HR26/cHjk9gmHSefIz/ZdDjsjqalxVm7axfxVW5m/agsLVm9ldWTO76yMECMLcxlTFA6P4/t1oVNWRmsdrUigFBaSGtxhdwVsWxN+bF8beb72s+f7640jlZndcIjUPs/uDmZs3FEZOfPYStnqLSz5dAfVNY4ZDO6RczA8Sory6J3bPpjjF4kzhYW0De6wd2udIFlbL1TWQOW2Q7dJz4LOhXVCpA907svejr1ZsrsT8zZmMn/Ndt5evZXdkS67vTpnHeyue0K/LhzXsxNp6rIrKUBhIVKrcsdnQbJ9LWxbfeiZye5644qF0qFTb2o692F7uwJWV+Xx3u7OzNvcgcW7O7POu9K+XRaj+3WJ9LoKd9ntkJkezPGJtIDCQiRaB/aGr43Uhkj9s5Sd66gzBiaOsT0jn09quvLh/jzKvRvryCc9rx89+g7k2IHHMXpAL/Jz1GVXEl+0YaFfhUQy2kO3geFHQ6r2w45PDoaIbVtL7rY15G5fy5Ctq2HHm4S8CnYA74UfFd6ZpWndOZBdSGbXfmRl50IoDSwNC6XhoTTM0iAt8jMUXm6hyHNLw9IiyywUeT/94DoWSiOUFl5GKEQolP7Z+qF0QgfXCS8PRZ6HQp9tQ2S/h/5UrzBpmMJCpDHpmZDXP/yoJwTh7r4718G2NRzYspqNa1ewY91H+LY1dN7xPgXbX6WdHWj1spurihA1kUd1nefh14feHe91ZyKwQ1/XbbMw6r/X8Hqfrd3A5x9c9/BrRYev1/D1JLfo1ks2FR0HUfKDR+O6D4WFSEuF0iIXzAvJ6DeB3qM/m+XL3fm4Yhfb9uyF6mq8phqvrqKmpgZqqvCaGmpqqvCa6oOvPfKz9jVe+37te9XgNVBTffB96i7zaqym+rP1Iq/xyKOmBqt97jVYTXXk9WfrWu1zD69b+xP/bFpcq/81X+elHdJsV3/dz57bUZvBD33vkM+os51RP5iOsK/D6m2dJvjD9hsHVZ36NL5SCyksROLIzBjQPYfw9C0iyUsNlCIi0iiFhYiINEphISIijVJYiIhIoxQWIiLSKIWFiIg0SmEhIiKNUliIiEijUmYgQTOrAFa34CO6AZtiVE6QUuU4QMeSqFLlWFLlOKBlx9LP3fMbWyllwqKlzKwsmpEXE12qHAfoWBJVqhxLqhwHtM6xqBlKREQapbAQEZFGKSw+c2fQBcRIqhwH6FgSVaocS6ocB7TCseiahYiINEpnFiIi0iiFRYSZ3WRmi8zsXTN72sx6BV1Tc5nZr8xseeR4HjWz3KBrai4z+7KZLTGzGjNLup4rZjbVzN43sxVmdl3Q9bSEmd1jZhvN7L2ga2kJM+tjZi+Y2bLIv62rgq6pucwsy8zeMrOFkWP577jtS81QYWbWyd13RJ5fCQx198sCLqtZzOwM4Hl3rzKzXwC4+7UBl9UsZjYEqAHuAP7d3csCLilqZpYGfACcDpQD84EZ7r400MKaycwmAbuAv7r78KDraS4zKwAK3P1tM8sBFgDnJOPfi5kZ0NHdd5lZBvAqcJW7vxHrfenMIqI2KCI60tDUwEnC3Z9296rIyzeAwiDraQl3X+bu7wddRzONBVa4+0p33w/MBM4OuKZmc/eXgS1B19FS7r7O3d+OPN8JLOOzmXCTioftirzMiDzi8t2lsKjDzG42s7XAhcB/Bl1PjHwTeDLoItqo3sDaOq/LSdIvpVRlZkXAaODNYCtpPjNLM7N3gY3AM+4el2NpU2FhZs+a2XsNPM4GcPcfu3sf4H7g8mCrPbrGjiWyzo+BKsLHk7CiOZYkZQ0sS9oz1lRjZtnAP4Dv1WtZSCruXu3uowi3IIw1s7g0EabH40MTlbt/PspV/wbMBm6IYzkt0tixmNlFQClwmif4hakm/L0km3KgT53XhcCnAdUidUTa9/8B3O/ujwRdTyy4+zYzexGYCsS8E0KbOrM4GjMbWOflWcDyoGppKTObClwLnOXue4Kupw2bDww0s/5mlglcADwecE1tXuSi8N3AMnf/TdD1tISZ5df2djSz9sDnidN3l3pDRZjZP4DBhHverAYuc/dPgq2qecxsBdAO2BxZ9EYS9+z6IvA/QD6wDXjX3acEW1X0zGwacCuQBtzj7jcHXFKzmdkDwCmERzjdANzg7ncHWlQzmNnJwCvAYsL/3wH+w93nBFdV85hZMfAXwv++QsCD7n5jXPalsBARkcaoGUpERBqlsBARkUYpLEREpFEKCxERaZTCQkREGqWwEGkCM9vV+FpH3f5hMxsQeZ5tZneY2UeREUNfNrNxZpYZed6mbpqVxKawEGklZjYMSHP3lZFFdxEemG+guw8DLga6RQYdfA44P5BCRRqgsBBpBgv7VWQMq8Vmdn5kecjM/hA5U5hlZnPM7LzIZhcCj0XWOwYYB1zv7jUAkdFpZ0fW/WdkfZGEoNNckeb5EjAKGEn4jub5ZvYycBJQBIwAuhMe/vqeyDYnAQ9Eng8jfDd69RE+/z1gTFwqF2kGnVmINM/JwAORET83AC8R/nI/GXjI3WvcfT3wQp1tCoCKaD48EiL7I5PziAROYSHSPA0NP3605QB7gazI8yXASDM72v/BdkBlM2oTiTmFhUjzvAycH5l4Jh+YBLxFeFrLcyPXLnoQHniv1jLgWAB3/wgoA/47MgoqZjawdg4PM+sKVLj7gdY6IJGjUViINM+jwCJgIfA88MNIs9M/CM9j8R7hecPfBLZHtpnNoeHxLaAnsMLMFgN/4rP5Lk4Fkm4UVEldGnVWJMbMLNvdd0XODt4CTnL39ZH5Bl6IvD7She3az3gE+FESzz8uKUa9oURib1ZkQppM4KbIGQfuvtfMbiA8D/eaI20cmSjpnwoKSSQ6sxARkUbpmoWIiDRKYSEiIo1SWIiISKMUFiIi0iiFhYiINEphISIijfr/BmGffa8wCMsAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0xe2b7e48>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "logloss_s1 =np.array(logloss_s).reshape(len(Cs),len(penaltys))\n",
    "x_axis = np.log10(Cs)\n",
    "for j, onePenalty in enumerate(penaltys):\n",
    "    pyplot.plot(x_axis, np.array(logloss_s1[:,j]), label = ' Test-' + onePenalty)\n",
    "\n",
    "pyplot.legend()\n",
    "pyplot.xlabel( 'log(C)' )                                                                                                      \n",
    "pyplot.ylabel( 'logloss' )\n",
    "\n",
    "pyplot.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1000.000000  pen l2 loss 0.428547 \n"
     ]
    }
   ],
   "source": [
    "bestCs = logloss_s1.argmin(axis = 0)\n",
    "\n",
    "best_logloss = logloss_s1[bestCs[0],0]\n",
    "best_penalty_index = 0\n",
    "best_penalty = penaltys[best_penalty_index]\n",
    "\n",
    "for j, onePenalty in enumerate(penaltys):\n",
    "    if logloss_s1[bestCs[j],j] < best_logloss:\n",
    "        best_logloss = logloss_s1[bestCs[j],j]\n",
    "        best_penalty_index = j\n",
    "        best_penalty = penaltys[best_penalty_index]\n",
    "\n",
    "bestC = Cs[bestCs[best_penalty_index]]\n",
    "\n",
    "print(\"%f  pen %s loss %f \"%(bestC,best_penalty,best_logloss)  )\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=5, error_score='raise',\n",
       "       estimator=LogisticRegression(C=1.0, class_weight=None, dual=False, fit_intercept=True,\n",
       "          intercept_scaling=1, max_iter=100, multi_class='ovr', n_jobs=1,\n",
       "          penalty='l2', random_state=None, solver='liblinear', tol=0.0001,\n",
       "          verbose=0, warm_start=False),\n",
       "       fit_params=None, iid=True, n_jobs=1,\n",
       "       param_grid={'penalty': ['l1', 'l2'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]},\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring='neg_log_loss', verbose=0)"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "\n",
    "#需要调优的参数\n",
    "# 请尝试将L1正则和L2正则分开，并配合合适的优化求解算法（slover）\n",
    "#tuned_parameters = {'penalty':['l1','l2'],\n",
    "#                   'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "#                   }\n",
    "penaltys = ['l1','l2']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "tuned_parameters = dict(penalty = penaltys, C = Cs)\n",
    "\n",
    "lr_penalty= LogisticRegression()\n",
    "grid= GridSearchCV(lr_penalty, tuned_parameters,cv=5, scoring='neg_log_loss')\n",
    "grid.fit(train_x_part,train_y_part)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'mean_fit_time': array([0.00379992, 0.00339994, 0.00279999, 0.00359998, 0.0046    ,\n",
       "        0.00419998, 0.0046    , 0.00400004, 0.00439997, 0.00459995,\n",
       "        0.00540009, 0.00459991, 0.00519996, 0.00439997]),\n",
       " 'mean_score_time': array([0.00360003, 0.00320001, 0.00320001, 0.00259995, 0.00279994,\n",
       "        0.00200005, 0.00300002, 0.00239997, 0.00240002, 0.00280008,\n",
       "        0.00320001, 0.00340009, 0.00359998, 0.004     ]),\n",
       " 'mean_test_score': array([-0.69314718, -0.64808512, -0.68093773, -0.54292008, -0.50555422,\n",
       "        -0.4979122 , -0.49746566, -0.49743547, -0.49785932, -0.49786951,\n",
       "        -0.49791945, -0.49792128, -0.49792396, -0.49792655]),\n",
       " 'mean_train_score': array([-0.69314718, -0.64694436, -0.67967583, -0.53726035, -0.49439124,\n",
       "        -0.48519925, -0.48160473, -0.48146364, -0.481403  , -0.48140155,\n",
       "        -0.4814009 , -0.48140088, -0.48140088, -0.48140088]),\n",
       " 'param_C': masked_array(data=[0.001, 0.001, 0.01, 0.01, 0.1, 0.1, 1, 1, 10, 10, 100,\n",
       "                    100, 1000, 1000],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'param_penalty': masked_array(data=['l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1', 'l2', 'l1',\n",
       "                    'l2', 'l1', 'l2', 'l1', 'l2'],\n",
       "              mask=[False, False, False, False, False, False, False, False,\n",
       "                    False, False, False, False, False, False],\n",
       "        fill_value='?',\n",
       "             dtype=object),\n",
       " 'params': [{'C': 0.001, 'penalty': 'l1'},\n",
       "  {'C': 0.001, 'penalty': 'l2'},\n",
       "  {'C': 0.01, 'penalty': 'l1'},\n",
       "  {'C': 0.01, 'penalty': 'l2'},\n",
       "  {'C': 0.1, 'penalty': 'l1'},\n",
       "  {'C': 0.1, 'penalty': 'l2'},\n",
       "  {'C': 1, 'penalty': 'l1'},\n",
       "  {'C': 1, 'penalty': 'l2'},\n",
       "  {'C': 10, 'penalty': 'l1'},\n",
       "  {'C': 10, 'penalty': 'l2'},\n",
       "  {'C': 100, 'penalty': 'l1'},\n",
       "  {'C': 100, 'penalty': 'l2'},\n",
       "  {'C': 1000, 'penalty': 'l1'},\n",
       "  {'C': 1000, 'penalty': 'l2'}],\n",
       " 'rank_test_score': array([14, 12, 13, 11, 10,  5,  2,  1,  3,  4,  6,  7,  8,  9]),\n",
       " 'split0_test_score': array([-0.69314718, -0.65002563, -0.67829107, -0.55090155, -0.52012587,\n",
       "        -0.50826448, -0.50771417, -0.50638103, -0.5066175 , -0.50649593,\n",
       "        -0.50652465, -0.50651248, -0.50651148, -0.50651419]),\n",
       " 'split0_train_score': array([-0.69314718, -0.64572623, -0.67663482, -0.53390451, -0.48949344,\n",
       "        -0.48195838, -0.47854574, -0.47839872, -0.47834278, -0.47834127,\n",
       "        -0.47834068, -0.47834066, -0.47834066, -0.47834066]),\n",
       " 'split1_test_score': array([-0.69314718, -0.64436414, -0.68786637, -0.52364861, -0.47109007,\n",
       "        -0.4604794 , -0.45390406, -0.45402081, -0.45358852, -0.45361518,\n",
       "        -0.45357537, -0.45357901, -0.45357531, -0.45357544]),\n",
       " 'split1_train_score': array([-0.69314718, -0.64911756, -0.68895798, -0.54438352, -0.50499686,\n",
       "        -0.4948011 , -0.4914652 , -0.49131649, -0.49126117, -0.49125962,\n",
       "        -0.49125903, -0.49125901, -0.49125901, -0.49125901]),\n",
       " 'split2_test_score': array([-0.69314718, -0.6493777 , -0.67813891, -0.54147085, -0.49306842,\n",
       "        -0.47983719, -0.47231904, -0.472463  , -0.47192004, -0.47195414,\n",
       "        -0.47190552, -0.47190786, -0.47190124, -0.47190328]),\n",
       " 'split2_train_score': array([-0.69314718, -0.64647148, -0.67679612, -0.53944064, -0.50132309,\n",
       "        -0.49094039, -0.48769494, -0.48753633, -0.48748172, -0.48748006,\n",
       "        -0.48747948, -0.48747946, -0.48747945, -0.48747945]),\n",
       " 'split3_test_score': array([-0.69314718, -0.65142902, -0.67536986, -0.56908844, -0.56352205,\n",
       "        -0.56711743, -0.58343274, -0.58508054, -0.58807993, -0.58825934,\n",
       "        -0.58858133, -0.58860061, -0.58862916, -0.58863498]),\n",
       " 'split3_train_score': array([-0.69314718, -0.64486658, -0.66966301, -0.52608539, -0.47586508,\n",
       "        -0.46594819, -0.46129308, -0.46118026, -0.46109745, -0.46109632,\n",
       "        -0.46109542, -0.46109541, -0.4610954 , -0.4610954 ]),\n",
       " 'split4_test_score': array([-0.69314718, -0.64521719, -0.68503197, -0.52952994, -0.48011062,\n",
       "        -0.47414778, -0.47035348, -0.46964588, -0.46952255, -0.46945669,\n",
       "        -0.46944601, -0.4694423 , -0.46943862, -0.46944091]),\n",
       " 'split4_train_score': array([-0.69314718, -0.64853995, -0.6863272 , -0.5424877 , -0.50027774,\n",
       "        -0.4923482 , -0.48902471, -0.48888641, -0.48883187, -0.48883048,\n",
       "        -0.48882989, -0.48882988, -0.48882987, -0.48882987]),\n",
       " 'std_fit_time': array([1.32658252e-03, 7.99965864e-04, 7.48264501e-04, 4.89862441e-04,\n",
       "        7.99989707e-04, 4.00066376e-04, 7.99989707e-04, 1.16800773e-07,\n",
       "        4.89920847e-04, 7.99906259e-04, 4.89862441e-04, 7.99942017e-04,\n",
       "        9.79802753e-04, 4.89920847e-04]),\n",
       " 'std_score_time': array([7.99965864e-04, 3.99971008e-04, 3.99971008e-04, 4.89881921e-04,\n",
       "        7.48379193e-04, 9.53674316e-08, 1.09545278e-03, 4.89940316e-04,\n",
       "        4.89901382e-04, 1.16619581e-03, 3.99971008e-04, 8.00013548e-04,\n",
       "        7.99942017e-04, 6.32409699e-04]),\n",
       " 'std_test_score': array([0.        , 0.00277937, 0.00470014, 0.01610512, 0.03329206,\n",
       "        0.03783697, 0.0462874 , 0.04688751, 0.04816423, 0.04822572,\n",
       "        0.04836049, 0.04836674, 0.04837917, 0.04838092]),\n",
       " 'std_train_score': array([0.        , 0.00163054, 0.00704551, 0.00661755, 0.01060554,\n",
       "        0.01055813, 0.01105709, 0.01104412, 0.01105444, 0.01105429,\n",
       "        0.0110544 , 0.0110544 , 0.0110544 , 0.0110544 ])}"
      ]
     },
     "execution_count": 84,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "grid.cv_results_"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.4974354725603503\n",
      "{'penalty': 'l2', 'C': 1}\n"
     ]
    }
   ],
   "source": [
    "print(-grid.best_score_)\n",
    "print(grid.best_params_)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.429425359060133\n"
     ]
    }
   ],
   "source": [
    "penaltys = ['l1']\n",
    "Cs = [0.001, 0.01, 0.1, 1, 10, 100, 1000]\n",
    "\n",
    "LR = LogisticRegression(penalty='l2', C=1)\n",
    "LR.fit(train_x_part,train_y_part)\n",
    "test_y_pred=LR.predict_proba(test_x)\n",
    "logloss_test = log_loss(test_y,test_y_pred)\n",
    "print(logloss_test)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
